<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   Spark MLlib实现的中文文本分类–Native Bayes  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/21613.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466460825: Accept with keywords: (title(0.5):中文,社区,分类,数盟,Spark,文本, topn(0.533333333333):社区,数盟,数组,官网,语料,人工智能,模型,准确率,文档,苹果,计算,文章,词频,Spark,数据,香蕉,中文,词库,词语,语料库,编号,算法,格式,spark,分词,分类,测试数据,特征,字段,文本).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    Spark MLlib实现的中文文本分类–Native Bayes
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    Spark MLlib实现的中文文本分类–Native Bayes | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,80] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech">
       文章
      </a>
      &gt;
      <a href="http://dataunion.org/category/tech/ai">
       人工智能
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/21613.html">
          Spark MLlib实现的中文文本分类–Native Bayes
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          883 次阅读 -
         </em>
         <a href="http://dataunion.org/category/tech/ai" rel="category tag">
          人工智能
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        文本分类是指将一篇文章归到事先定义好的某一类或者某几类，在数据平台的一个典型的应用场景是，通过爬取用户浏览过的页面内容，识别出用户的浏览偏好，从而丰富该用户的画像。
        <br/>
        本文介绍使用Spark MLlib提供的朴素贝叶斯（Native Bayes）算法，完成对中文文本的分类过程。主要包括中文分词、文本表示（TF-IDF）、模型训练、分类预测等。
       </p>
       <h2>
        中文分词
       </h2>
       <p>
        对于中文文本分类而言，需要先对文章进行分词，我使用的是IKAnalyzer中文分析工具，其中自己可以配置扩展词库来使分词结果更合理，我从搜狗、百度输入法下载了细胞词库，将其作为扩展词库。这里不再介绍分词。
       </p>
       <h2>
        中文词语特征值转换（TF-IDF）
       </h2>
       <p>
        分好词后，每一个词都作为一个特征，但需要将中文词语转换成Double型来表示，通常使用该词语的TF-IDF值作为特征值，Spark提供了全 面的特征抽取及转换的API，非常方便，详见http://spark.apache.org/docs/latest/ml- features.html,这里介绍下TF-IDF的API:
       </p>
       <p>
        比如，训练语料/tmp/lxw1234/1.txt：
       </p>
       <p>
        0,苹果 官网 苹果 宣布
        <br/>
        1,苹果 梨 香蕉
       </p>
       <p>
        逗号分隔的第一列为分类编号，0为科技，1为水果。
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a94106da096665355" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <span class="crayon-language">
           Scala
          </span>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          case class RawDataRecord(category: String, text: String)
 
val conf = new SparkConf().setMaster("yarn-client")
val sc = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.implicits._
 
//将原始数据映射到DataFrame中，字段category为分类编号，字段text为分好的词，以空格分隔
var srcDF = sc.textFile("/tmp/lxw1234/1.txt").map {
x =&gt;
var data = x.split(",")
RawDataRecord(data(0),data(1))
}.toDF()
 
srcDF.select("category", "text").take(2).foreach(println)
[0,苹果 官网 苹果 宣布]
[1,苹果 梨 香蕉]
//将分好的词转换为数组
var tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words")
var wordsData = tokenizer.transform(srcDF)
 
wordsData.select($"category",$"text",$"words").take(2).foreach(println)
[0,苹果 官网 苹果 宣布,WrappedArray(苹果, 官网, 苹果, 宣布)]
[1,苹果 梨 香蕉,WrappedArray(苹果, 梨, 香蕉)]
 
//将每个词转换成Int型，并计算其在文档中的词频（TF）
var hashingTF =
new HashingTF().setInputCol("words").setOutputCol("rawFeatures").setNumFeatures(100)
var featurizedData = hashingTF.transform(wordsData)
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-4">
               4
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-5">
               5
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-6">
               6
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-7">
               7
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-8">
               8
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-9">
               9
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-10">
               10
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-11">
               11
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-12">
               12
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-13">
               13
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-14">
               14
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-15">
               15
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-16">
               16
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-17">
               17
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-18">
               18
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-19">
               19
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-20">
               20
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-21">
               21
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-22">
               22
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-23">
               23
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-24">
               24
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-25">
               25
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-26">
               26
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-27">
               27
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106da096665355-28">
               28
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106da096665355-29">
               29
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-57686a94106da096665355-1">
               <span class="crayon-st">
                case
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                class
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                RawDataRecord
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                category
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                text
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-2">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-3">
               <span class="crayon-r">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                conf
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                SparkConf
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setMaster
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "yarn-client"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-4">
               <span class="crayon-r">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sc
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                SparkContext
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                conf
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-5">
               <span class="crayon-r">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sqlContext
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                org
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                apache
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                spark
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                sql
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                SQLContext
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                sc
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-6">
               <span class="crayon-r">
                import
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sqlContext
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                implicits
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-r">
                _
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-7">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-8">
               <span class="crayon-c">
                //将原始数据映射到DataFrame中，字段category为分类编号，字段text为分好的词，以空格分隔
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-9">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                srcDF
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                sc
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                textFile
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "/tmp/lxw1234/1.txt"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-i">
                map
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-10">
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-o">
                &gt;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-11">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                data
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                x
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                split
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                ","
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-12">
               <span class="crayon-e">
                RawDataRecord
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e">
                data
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-e">
                data
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-13">
               <span class="crayon-sy">
                }
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                toDF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-14">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-15">
               <span class="crayon-v">
                srcDF
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "category"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                "text"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-st">
                foreach
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                println
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-16">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ,
               </span>
               苹果
               <span class="crayon-h">
               </span>
               官网
               <span class="crayon-h">
               </span>
               苹果
               <span class="crayon-h">
               </span>
               宣布
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-17">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               苹果
               <span class="crayon-h">
               </span>
               梨
               <span class="crayon-h">
               </span>
               香蕉
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-18">
               <span class="crayon-c">
                //将分好的词转换为数组
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-19">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                tokenizer
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Tokenizer
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "text"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-20">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                wordsData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                tokenizer
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                srcDF
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-21">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-22">
               <span class="crayon-v">
                wordsData
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "category"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "text"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-st">
                foreach
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                println
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-23">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ,
               </span>
               苹果
               <span class="crayon-h">
               </span>
               官网
               <span class="crayon-h">
               </span>
               苹果
               <span class="crayon-h">
               </span>
               宣布
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-e">
                WrappedArray
               </span>
               <span class="crayon-sy">
                (
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               官网
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               宣布
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-24">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               苹果
               <span class="crayon-h">
               </span>
               梨
               <span class="crayon-h">
               </span>
               香蕉
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-e">
                WrappedArray
               </span>
               <span class="crayon-sy">
                (
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               梨
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               香蕉
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-25">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-26">
               <span class="crayon-c">
                //将每个词转换成Int型，并计算其在文档中的词频（TF）
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-27">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                hashingTF
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106da096665355-28">
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                HashingTF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "rawFeatures"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setNumFeatures
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                100
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106da096665355-29">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                featurizedData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                hashingTF
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                wordsData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.0370 seconds] -->
       <p>
       </p>
       <p>
        这里将中文词语转换成INT型的Hashing算法，类似于Bloomfilter，上面的setNumFeatures(100)表示将Hash 分桶的数量设置为100个，这个值默认为2的20次方，即1048576，可以根据你的词语数量来调整，一般来说，这个值越大，不同的词被计算为一个 Hash值的概率就越小，数据也更准确，但需要消耗更大的内存，和Bloomfilter是一个道理。
       </p>
       <ol class="linenums">
        <li class="L0">
         <span class="pln">
          featurizedData
         </span>
         <span class="pun">
          .
         </span>
         <span class="kwd">
          select
         </span>
         <span class="pun">
          (
         </span>
         <span class="pln">
          $
         </span>
         <span class="str">
          “category”
         </span>
         <span class="pun">
          ,
         </span>
         <span class="pln">
          $
         </span>
         <span class="str">
          “words”
         </span>
         <span class="pun">
          ,
         </span>
         <span class="pln">
          $
         </span>
         <span class="str">
          “rawFeatures”
         </span>
         <span class="pun">
          ).
         </span>
         <span class="pln">
          take
         </span>
         <span class="pun">
          (
         </span>
         <span class="lit">
          2
         </span>
         <span class="pun">
          ).
         </span>
         <span class="kwd">
          foreach
         </span>
         <span class="pun">
          (
         </span>
         <span class="pln">
          println
         </span>
         <span class="pun">
          )
         </span>
        </li>
        <li class="L1">
         <span class="pun">
          [
         </span>
         <span class="lit">
          0
         </span>
         <span class="pun">
          ,
         </span>
         <span class="typ">
          WrappedArray
         </span>
         <span class="pun">
          (苹果,
         </span>
         <span class="pun">
          官网,
         </span>
         <span class="pun">
          苹果,
         </span>
         <span class="pun">
          宣布),(
         </span>
         <span class="lit">
          100
         </span>
         <span class="pun">
          ,[
         </span>
         <span class="lit">
          23
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          81
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          96
         </span>
         <span class="pun">
          ],[
         </span>
         <span class="lit">
          2.0
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          1.0
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          1.0
         </span>
         <span class="pun">
          ])]
         </span>
        </li>
        <li class="L2">
         <span class="pun">
          [
         </span>
         <span class="lit">
          1
         </span>
         <span class="pun">
          ,
         </span>
         <span class="typ">
          WrappedArray
         </span>
         <span class="pun">
          (苹果,
         </span>
         <span class="pun">
          梨,
         </span>
         <span class="pun">
          香蕉),(
         </span>
         <span class="lit">
          100
         </span>
         <span class="pun">
          ,[
         </span>
         <span class="lit">
          23
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          72
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          92
         </span>
         <span class="pun">
          ],[
         </span>
         <span class="lit">
          1.0
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          1.0
         </span>
         <span class="pun">
          ,
         </span>
         <span class="lit">
          1.0
         </span>
         <span class="pun">
          ])]
         </span>
        </li>
       </ol>
       <p>
        结果中，“苹果”用23来表示，第一个文档中，词频为2，第二个文档中词频为1.
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a94106ee760435565" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <span class="crayon-language">
           Scala
          </span>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          //计算TF-IDF值
var idf = new IDF().setInputCol("rawFeatures").setOutputCol("features")
var idfModel = idf.fit(featurizedData)
var rescaledData = idfModel.transform(featurizedData)
rescaledData.select($"category", $"words", $"features").take(2).foreach(println)
 
[0,WrappedArray(苹果, 官网, 苹果, 宣布),(100,[23,81,96],[0.0,0.4054651081081644,0.4054651081081644])]
[1,WrappedArray(苹果, 梨, 香蕉),(100,[23,72,92],[0.0,0.4054651081081644,0.4054651081081644])]
 
//因为一共只有两个文档，且都出现了“苹果”，因此该词的TF-IDF值为0.
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-57686a94106ee760435565-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106ee760435565-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106ee760435565-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106ee760435565-4">
               4
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106ee760435565-5">
               5
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106ee760435565-6">
               6
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106ee760435565-7">
               7
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106ee760435565-8">
               8
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106ee760435565-9">
               9
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106ee760435565-10">
               10
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-57686a94106ee760435565-1">
               <span class="crayon-c">
                //计算TF-IDF值
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106ee760435565-2">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                idf
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-r">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                IDF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "rawFeatures"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "features"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106ee760435565-3">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                idfModel
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                idf
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                fit
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                featurizedData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106ee760435565-4">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                rescaledData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                idfModel
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                featurizedData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106ee760435565-5">
               <span class="crayon-v">
                rescaledData
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "category"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "features"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-st">
                foreach
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                println
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106ee760435565-6">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106ee760435565-7">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-e">
                WrappedArray
               </span>
               <span class="crayon-sy">
                (
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               官网
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               宣布
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                100
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                23
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                81
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                96
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0.0
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                0.4054651081081644
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                0.4054651081081644
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106ee760435565-8">
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                1
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-e">
                WrappedArray
               </span>
               <span class="crayon-sy">
                (
               </span>
               苹果
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               梨
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               香蕉
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-cn">
                100
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                23
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                72
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                92
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                [
               </span>
               <span class="crayon-cn">
                0.0
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                0.4054651081081644
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-cn">
                0.4054651081081644
               </span>
               <span class="crayon-sy">
                ]
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                ]
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106ee760435565-9">
               <span class="crayon-h">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106ee760435565-10">
               <span class="crayon-c">
                //因为一共只有两个文档，且都出现了“苹果”，因此该词的TF-IDF值为0.
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.0173 seconds] -->
       <p>
       </p>
       <ol class="linenums">
        <li class="L0">
         <span class="pln">
         </span>
        </li>
       </ol>
       <p>
        最后一步，将上面的数据转换成Bayes算法需要的格式，如：
       </p>
       <p>
        https://github.com/apache/spark/blob/branch-1.5/data/mllib/sample_naive_bayes_data.txt
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a94106fa725053588" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <span class="crayon-language">
           Scala
          </span>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          var trainDataRdd = rescaledData.select($"category",$"features").map {
case Row(label: String, features: Vector) =&gt;
LabeledPoint(label.toDouble, Vectors.dense(features.toArray))
}
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-57686a94106fa725053588-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106fa725053588-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-57686a94106fa725053588-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a94106fa725053588-4">
               4
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-57686a94106fa725053588-1">
               <span class="crayon-r">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                trainDataRdd
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                rescaledData
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "category"
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-sy">
                $
               </span>
               <span class="crayon-s">
                "features"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-i">
                map
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106fa725053588-2">
               <span class="crayon-st">
                case
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Row
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                label
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                features
               </span>
               <span class="crayon-o">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Vector
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-o">
                =
               </span>
               <span class="crayon-o">
                &gt;
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a94106fa725053588-3">
               <span class="crayon-e">
                LabeledPoint
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                label
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                toDouble
               </span>
               <span class="crayon-sy">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-v">
                Vectors
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-e">
                dense
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-v">
                features
               </span>
               <span class="crayon-sy">
                .
               </span>
               <span class="crayon-v">
                toArray
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a94106fa725053588-4">
               <span class="crayon-sy">
                }
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.0106 seconds] -->
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-1.jpg"/>
       </p>
       <p>
        每一个LabeledPoint中，特征数组的长度为100（setNumFeatures(100)），”官网”和”宣布”对应的特征索引号分别为81和96，因此，在特征数组中，第81位和第96位分别为它们的TF-IDF值。
       </p>
       <p>
        到此，中文词语特征表示的工作已经完成，trainDataRdd已经可以作为Bayes算法的输入了。
       </p>
       <h2>
        分类模型训练
       </h2>
       <p>
        训练模型，语料非常重要，我这里使用的是搜狗提供的分类语料库，很早之前的了，这里只作为学习测试使用。
       </p>
       <p>
        下载地址在：http://www.sogou.com/labs/dl/c.html，语料库一共有10个分类：
       </p>
       <p>
        C000007 汽车
        <br/>
        C000008 财经
        <br/>
        C000010  IT
        <br/>
        C000013 健康
        <br/>
        C000014 体育
        <br/>
        C000016 旅游
        <br/>
        C000020 教育
        <br/>
        C000022 招聘
        <br/>
        C000023 文化
        <br/>
        C000024 军事
       </p>
       <p>
        每个分类下有几千个文档，这里将这些语料进行分词，然后每一个分类生成一个文件，在该文件中，每一行数据表示一个文档的分词结果，重新用0-9作为这10个分类的编号：
        <br/>
        0 汽车
        <br/>
        1 财经
        <br/>
        2 IT
        <br/>
        3 健康
        <br/>
        4 体育
        <br/>
        5 旅游
        <br/>
        6 教育
        <br/>
        7 招聘
        <br/>
        8 文化
        <br/>
        9 军事
       </p>
       <p>
        比如，汽车分类下的文件内容为：
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-2.jpg"/>
       </p>
       <p>
        数据准备好了，接下来进行模型训练及分类预测,代码：
       </p>
       <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
       <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686a9410705234171990" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
        <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
         <span class="crayon-title">
         </span>
         <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
          <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-wrap-button" title="切换自动换行">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-expand-button" title="点击展开代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-copy-button" title="复制代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
           <div class="crayon-button-icon">
           </div>
          </div>
          <span class="crayon-language">
           Sass
          </span>
         </div>
        </div>
        <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
        </div>
        <div class="crayon-plain-wrap">
         <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
          package com.lxw1234.textclassification
 
import scala.reflect.runtime.universe
 
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.ml.feature.HashingTF
import org.apache.spark.ml.feature.IDF
import org.apache.spark.ml.feature.Tokenizer
import org.apache.spark.mllib.classification.NaiveBayes
import org.apache.spark.mllib.linalg.Vector
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.sql.Row
 
 
object TestNativeBayes {
case class RawDataRecord(category: String, text: String)
def main(args : Array[String]) {
val conf = new SparkConf().setMaster("yarn-client")
val sc = new SparkContext(conf)
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
import sqlContext.implicits._
var srcRDD = sc.textFile("/tmp/lxw1234/sougou/").map {
x =&gt;
var data = x.split(",")
RawDataRecord(data(0),data(1))
}
//70%作为训练数据，30%作为测试数据
val splits = srcRDD.randomSplit(Array(0.7, 0.3))
var trainingDF = splits(0).toDF()
var testDF = splits(1).toDF()
//将词语转换成数组
var tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words")
var wordsData = tokenizer.transform(trainingDF)
println("output1：")
wordsData.select($"category",$"text",$"words").take(1)
//计算每个词在文档中的词频
var hashingTF = new HashingTF().setNumFeatures(500000).setInputCol("words").setOutputCol("rawFeatures")
var featurizedData = hashingTF.transform(wordsData)
println("output2：")
featurizedData.select($"category", $"words", $"rawFeatures").take(1)
//计算每个词的TF-IDF
var idf = new IDF().setInputCol("rawFeatures").setOutputCol("features")
var idfModel = idf.fit(featurizedData)
var rescaledData = idfModel.transform(featurizedData)
println("output3：")
rescaledData.select($"category", $"features").take(1)
//转换成Bayes的输入格式
var trainDataRdd = rescaledData.select($"category",$"features").map {
case Row(label: String, features: Vector) =&gt;
LabeledPoint(label.toDouble, Vectors.dense(features.toArray))
}
println("output4：")
trainDataRdd.take(1)
//训练模型
val model = NaiveBayes.train(trainDataRdd, lambda = 1.0, modelType = "multinomial")
//测试数据集，做同样的特征表示及格式转换
var testwordsData = tokenizer.transform(testDF)
var testfeaturizedData = hashingTF.transform(testwordsData)
var testrescaledData = idfModel.transform(testfeaturizedData)
var testDataRdd = testrescaledData.select($"category",$"features").map {
case Row(label: String, features: Vector) =&gt;
LabeledPoint(label.toDouble, Vectors.dense(features.toArray))
}
//对测试数据集使用训练模型进行分类预测
val testpredictionAndLabel = testDataRdd.map(p =&gt; (model.predict(p.features), p.label))
//统计分类准确率
var testaccuracy = 1.0 * testpredictionAndLabel.filter(x =&gt; x._1 == x._2).count() / testDataRdd.count()
println("output5：")
println(testaccuracy)
}
}
         </textarea>
        </div>
        <div class="crayon-main" style="">
         <table class="crayon-table">
          <tbody>
           <tr class="crayon-row">
            <td class="crayon-nums " data-settings="show">
             <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-1">
               1
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-2">
               2
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-3">
               3
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-4">
               4
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-5">
               5
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-6">
               6
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-7">
               7
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-8">
               8
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-9">
               9
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-10">
               10
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-11">
               11
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-12">
               12
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-13">
               13
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-14">
               14
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-15">
               15
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-16">
               16
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-17">
               17
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-18">
               18
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-19">
               19
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-20">
               20
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-21">
               21
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-22">
               22
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-23">
               23
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-24">
               24
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-25">
               25
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-26">
               26
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-27">
               27
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-28">
               28
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-29">
               29
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-30">
               30
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-31">
               31
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-32">
               32
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-33">
               33
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-34">
               34
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-35">
               35
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-36">
               36
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-37">
               37
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-38">
               38
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-39">
               39
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-40">
               40
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-41">
               41
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-42">
               42
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-43">
               43
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-44">
               44
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-45">
               45
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-46">
               46
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-47">
               47
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-48">
               48
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-49">
               49
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-50">
               50
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-51">
               51
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-52">
               52
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-53">
               53
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-54">
               54
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-55">
               55
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-56">
               56
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-57">
               57
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-58">
               58
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-59">
               59
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-60">
               60
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-61">
               61
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-62">
               62
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-63">
               63
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-64">
               64
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-65">
               65
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-66">
               66
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-67">
               67
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-68">
               68
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-69">
               69
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-70">
               70
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-71">
               71
              </div>
              <div class="crayon-num crayon-striped-num" data-line="crayon-57686a9410705234171990-72">
               72
              </div>
              <div class="crayon-num" data-line="crayon-57686a9410705234171990-73">
               73
              </div>
             </div>
            </td>
            <td class="crayon-code">
             <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
              <div class="crayon-line" id="crayon-57686a9410705234171990-1">
               <span class="crayon-t">
                package
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-k ">
                com.lxw1234.textclassification
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-2">
               <span class="crayon-k ">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-3">
               <span class="crayon-k ">
                import scala.reflect.runtime.universe
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-4">
               <span class="crayon-k ">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-5">
               <span class="crayon-k ">
                import org.apache.spark.SparkConf
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-6">
               <span class="crayon-k ">
                import org.apache.spark.SparkContext
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-7">
               <span class="crayon-k ">
                import org.apache.spark.ml.feature.HashingTF
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-8">
               <span class="crayon-k ">
                import org.apache.spark.ml.feature.IDF
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-9">
               <span class="crayon-k ">
                import org.apache.spark.ml.feature.Tokenizer
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-10">
               <span class="crayon-k ">
                import org.apache.spark.mllib.classification.NaiveBayes
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-11">
               <span class="crayon-k ">
                import org.apache.spark.mllib.linalg.Vector
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-12">
               <span class="crayon-k ">
                import org.apache.spark.mllib.linalg.Vectors
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-13">
               <span class="crayon-k ">
                import org.apache.spark.mllib.regression.LabeledPoint
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-14">
               <span class="crayon-k ">
                import org.apache.spark.sql.Row
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-15">
               <span class="crayon-k ">
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-16">
               <span class="crayon-k ">
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-17">
               <span class="crayon-k ">
                object TestNativeBayes
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-18">
               <span class="crayon-i ">
                case
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                class
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                RawDataRecord
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e ">
                category
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-i ">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                text
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-19">
               <span class="crayon-i ">
                def
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                main
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e ">
                args
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                Array
               </span>
               <span class="crayon-i ">
                [String]
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-20">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                conf
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                SparkConf
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setMaster
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "yarn-client"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-21">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                sc
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                SparkContext
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                conf
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-22">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                sqlContext
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                org.apache.spark.sql.SQLContext
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                sc
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-23">
               <span class="crayon-i ">
                import
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                sqlContext.implicits._
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-24">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                srcRDD
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                sc.textFile
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "/tmp/lxw1234/sougou/"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-k ">
                .map
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-25">
               <span class="crayon-i ">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =&gt;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-26">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                data
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                x.split
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                ","
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-27">
               <span class="crayon-e">
                RawDataRecord
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e">
                data
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                0
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                ,data
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-28">
               <span class="crayon-sy">
                }
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-29">
               <span class="crayon-c">
                //70%作为训练数据，30%作为测试数据
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-30">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                splits
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                srcRDD.randomSplit
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e">
                Array
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                0.7,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                0.3
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-31">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                trainingDF
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                splits
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                0
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .toDF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-32">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testDF
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                splits
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .toDF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-33">
               <span class="crayon-c">
                //将词语转换成数组
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-34">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                tokenizer
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Tokenizer
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "text"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-35">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                wordsData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                tokenizer.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                trainingDF
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-36">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "output1："
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-37">
               <span class="crayon-i ">
                wordsData.select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                $"category",$"text",$"words"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-38">
               <span class="crayon-c">
                //计算每个词在文档中的词频
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-39">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                hashingTF
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                HashingTF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setNumFeatures
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                500000
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "words"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "rawFeatures"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-40">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                featurizedData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                hashingTF.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                wordsData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-41">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "output2："
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-42">
               <span class="crayon-i ">
                featurizedData.select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                $"category",
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                $"words",
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                $"rawFeatures"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-43">
               <span class="crayon-c">
                //计算每个词的TF-IDF
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-44">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                idf
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                new
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                IDF
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setInputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "rawFeatures"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .setOutputCol
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "features"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-45">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                idfModel
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                idf.fit
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                featurizedData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-46">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                rescaledData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                idfModel.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                featurizedData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-47">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "output3："
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-48">
               <span class="crayon-i ">
                rescaledData.select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                $"category",
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                $"features"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-49">
               <span class="crayon-c">
                //转换成Bayes的输入格式
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-50">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                trainDataRdd
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                rescaledData.select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                $"category",$"features"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-k ">
                .map
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-51">
               <span class="crayon-i ">
                case
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Row
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e ">
                label
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-i ">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                features
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                Vector
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =&gt;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-52">
               <span class="crayon-e">
                LabeledPoint
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                label.toDouble,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                Vectors.dense
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                features.toArray
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-53">
               <span class="crayon-sy">
                }
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-54">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "output4："
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-55">
               <span class="crayon-i ">
                trainDataRdd.take
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                1
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-56">
               <span class="crayon-c">
                //训练模型
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-57">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                model
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                NaiveBayes.train
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                trainDataRdd,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                lambda
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                1.0,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                modelType
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-s">
                "multinomial"
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-58">
               <span class="crayon-c">
                //测试数据集，做同样的特征表示及格式转换
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-59">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testwordsData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                tokenizer.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                testDF
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-60">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testfeaturizedData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                hashingTF.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                testwordsData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-61">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testrescaledData
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                idfModel.transform
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                testfeaturizedData
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-62">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testDataRdd
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testrescaledData.select
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                $"category",$"features"
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-k ">
                .map
               </span>
               <span class="crayon-sy">
                {
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-63">
               <span class="crayon-i ">
                case
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e">
                Row
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-e ">
                label
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-t">
                String
               </span>
               <span class="crayon-i ">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-e ">
                features
               </span>
               <span class="crayon-sy">
                :
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                Vector
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =&gt;
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-64">
               <span class="crayon-e">
                LabeledPoint
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                label.toDouble,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                Vectors.dense
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                features.toArray
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-65">
               <span class="crayon-sy">
                }
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-66">
               <span class="crayon-c">
                //对测试数据集使用训练模型进行分类预测
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-67">
               <span class="crayon-i ">
                val
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testpredictionAndLabel
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testDataRdd.map
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                p
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =&gt;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                model.predict
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                p.features
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                ,
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                p.label
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-68">
               <span class="crayon-c">
                //统计分类准确率
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-69">
               <span class="crayon-t">
                var
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testaccuracy
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                1.0
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                *
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testpredictionAndLabel.filter
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                x
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                =&gt;
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                x._1
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                ==
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                x._2
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-i ">
                .count
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                /
               </span>
               <span class="crayon-h">
               </span>
               <span class="crayon-i ">
                testDataRdd.count
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-70">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-s">
                "output5："
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-71">
               <span class="crayon-e">
                println
               </span>
               <span class="crayon-sy">
                (
               </span>
               <span class="crayon-i ">
                testaccuracy
               </span>
               <span class="crayon-sy">
                )
               </span>
              </div>
              <div class="crayon-line crayon-striped-line" id="crayon-57686a9410705234171990-72">
               <span class="crayon-sy">
                }
               </span>
              </div>
              <div class="crayon-line" id="crayon-57686a9410705234171990-73">
               <span class="crayon-sy">
                }
               </span>
              </div>
             </div>
            </td>
           </tr>
          </tbody>
         </table>
        </div>
       </div>
       <!-- [Format Time: 0.0612 seconds] -->
       <p>
       </p>
       <p>
        执行后，主要输出如下：
       </p>
       <p>
        output1：（将词语转换成数组）
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-3.jpg"/>
       </p>
       <p>
        output2：（计算每个词在文档中的词频）
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-4.jpg"/>
       </p>
       <p>
        output3：（计算每个词的TF-IDF）
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-5.jpg"/>
       </p>
       <p>
        output4：（Bayes算法的输入数据格式）
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-6.jpg"/>
       </p>
       <p>
        output5：（测试数据集分类准确率）
       </p>
       <p>
        <img src="http://7xipth.com1.z0.glb.clouddn.com/20160122-7.jpg"/>
       </p>
       <p>
        准确率90%，还可以。接下来需要收集分类更细，时间更新的数据来训练和测试了。。
       </p>
       <p>
       </p>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/24663.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/095356be8cb4z9ke9389kd-204x200.jpeg"/>
        </a>
        <a class="link" href="http://dataunion.org/24663.html" rel="bookmark" title="人工智能界最强大脑畅谈AI未来之路">
         人工智能界最强大脑畅谈AI未来之路
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24652.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/02/20150522141422337-265x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24652.html" rel="bookmark" title="重磅 | MIT与谷歌专家合著论文：机器学习和神经科学的相互启发与融合（附论文）">
         重磅 | MIT与谷歌专家合著论文：机器学习和神经科学的相互启发与融合（附论文）
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24616.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/224806wllbe6tcimyyi9xt-300x167.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24616.html" rel="bookmark" title="微软人工智能计划背后的华人力量">
         微软人工智能计划背后的华人力量
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24562.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/8aac7ceb921d1684f06e6f65e2c1f224-300x173.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24562.html" rel="bookmark" title="各位大佬，别再拿人工智能当春药了！">
         各位大佬，别再拿人工智能当春药了！
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F21613.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/21613.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/21613.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>